{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "W9_Tutorial1.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "0LMCVKg3EOSn",
        "UfitwfVdTERX",
        "oYgqtChAwJGr",
        "1UUH2KjZw_IX",
        "gbgTqgB5rH_W"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/CIS-522/course-content/blob/main/tutorials/W09_RNN/W9_Tutorial1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oBAUnAOfC268"
      },
      "source": [
        "# CIS-522 Week 9 Part 1\n",
        "# Introduction to Recurrent Neural Networks (RNNs)\n",
        "\n",
        "__Instructor__: Lyle Ungar\n",
        "\n",
        "__Content creators:__ Anushree Hede, Pooja Consul\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULqtMpvDtxl4",
        "cellView": "form"
      },
      "source": [
        "#@title Video: Intro\n",
        "import time\n",
        "try: t0;\n",
        "except NameError: t0=time.time()\n",
        "\n",
        "from IPython.display import YouTubeVideo\n",
        "video = YouTubeVideo(id=\"vSPUeMd6sy4\", width=854, height=480, fs=1)\n",
        "print(\"Video available at https://youtube.com/watch?v=\" + video.id)\n",
        "\n",
        "video"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "a5NQBmTeSs5S"
      },
      "source": [
        "#@markdown What is your Pennkey and pod? (text, not numbers, e.g. bfranklin)\n",
        "my_pennkey = '' #@param {type:\"string\"}\n",
        "my_pod = 'Select' #@param ['Select', 'euclidean-wombat', 'sublime-newt', 'buoyant-unicorn', 'lackadaisical-manatee','indelible-stingray','superfluous-lyrebird','discreet-reindeer','quizzical-goldfish','astute-jellyfish','ubiquitous-cheetah','nonchalant-crocodile','fashionable-lemur','spiffy-eagle','electric-emu','quotidian-lion']\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "jm3jlhwY1ixt"
      },
      "source": [
        "# @title Week 9 Slides\n",
        "from IPython.display import HTML\n",
        "HTML('<iframe src=\"https://docs.google.com/presentation/d/1TAE6fukLA_Pc6lAZgEOgt-fvg9p2uvEveM-PQVf6D7g/embed?start=false&loop=false&delayms=3000\" frameborder=\"0\" width=\"960\" height=\"569\" allowfullscreen=\"true\" mozallowfullscreen=\"true\" webkitallowfullscreen=\"true\"></iframe>')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0LMCVKg3EOSn"
      },
      "source": [
        "## Recap the experience from last week\n",
        "\n",
        "What did you learn last week. What questions do you have? [15 min discussion]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "0q2tL2Q5CAde"
      },
      "source": [
        "learning_from_previous_week = '' #@param {type:\"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x_ZIAAg0ckH-"
      },
      "source": [
        "*Estimated time: 20 minutes since start*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UfitwfVdTERX"
      },
      "source": [
        "---\n",
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UR8y41zOT4jz"
      },
      "source": [
        "# imports\n",
        "\n",
        "!pip install --upgrade gensim\n",
        "!pip install torchtext==0.4.0\n",
        "!pip install unidecode\n",
        "!pip install d2l\n",
        "\n",
        "import re\n",
        "import os\n",
        "import sys\n",
        "import math\n",
        "import time\n",
        "import nltk\n",
        "import torch\n",
        "import random\n",
        "import string\n",
        "import unidecode\n",
        "import collections\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch.nn as nn\n",
        "import matplotlib.cm as cm\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "\n",
        "\n",
        "from gensim.models import Word2Vec\n",
        "from nltk.corpus import brown\n",
        "from sklearn.manifold import TSNE\n",
        "from torch.autograd import Variable\n",
        "from torchtext import data, datasets\n",
        "from torchtext.vocab import Vectors\n",
        "\n",
        "from IPython.display import Image, YouTubeVideo\n",
        "from torch.nn import functional as F\n",
        "from d2l import torch as d2l\n",
        "\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('brown')\n",
        "nltk.download('webtext')\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ZEdeLHKe-05",
        "cellView": "form"
      },
      "source": [
        "# @title Seeds\n",
        "seed = 522\n",
        "random.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "torch.cuda.manual_seed_all(seed)\n",
        "torch.cuda.manual_seed(seed)\n",
        "np.random.seed(seed)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "torch.backends.cudnn.benchmark = False\n",
        "def seed_worker(worker_id):\n",
        "    worker_seed = torch.initial_seed() % 2**32\n",
        "    np.random.seed(worker_seed)\n",
        "    random.seed(worker_seed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2nk8O0txk01r",
        "cellView": "form"
      },
      "source": [
        "# @title Figure Settings\n",
        "import ipywidgets as widgets\n",
        "%matplotlib inline \n",
        "fig_w, fig_h = (8, 6)\n",
        "plt.rcParams.update({'figure.figsize': (fig_w, fig_h)})\n",
        "%config InlineBackend.figure_format = 'retina'\n",
        "SMALL_SIZE = 12\n",
        "\n",
        "plt.style.use(\"https://raw.githubusercontent.com/NeuromatchAcademy/\"\n",
        "              \"course-content/master/nma.mplstyle\")\n",
        "\n",
        "# plt.rcParams.update(plt.rcParamsDefault)\n",
        "# plt.rc('font', size=SMALL_SIZE)          # controls default text sizes\n",
        "# plt.rc('axes', titlesize=SMALL_SIZE)     # fontsize of the axes title\n",
        "# plt.rc('axes', labelsize=SMALL_SIZE)    # fontsize of the x and y labels\n",
        "# plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels\n",
        "# plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels\n",
        "# plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize\n",
        "# plt.rc('figure', titlesize=SMALL_SIZE)  # fontsize of the figure title"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "urklKPDcXXHb"
      },
      "source": [
        "---\n",
        "# Section 1: Memory"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZrEi84WAyYvF",
        "cellView": "form"
      },
      "source": [
        "#@title Video: Memory\n",
        "try: t1;\n",
        "except NameError: t1=time.time()\n",
        "\n",
        "video = YouTubeVideo(id=\"q4XN3XIYUto\", width=854, height=480, fs=1)\n",
        "print(\"Video available at https://youtube.com/watch?v=\" + video.id)\n",
        "\n",
        "video"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O2_bM01E7Akf"
      },
      "source": [
        "Discuss briefly and answer below: What are examples of machine learning time series problems where standard recurrent networks are likely to perform badly due to their short memory?  How else might one address the problem of long term memory other than neural turing machines?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kflny5yk7i-l",
        "cellView": "form"
      },
      "source": [
        "where_RNNs_fail = '' #@param {type:\"string\"}\n",
        "alternate_long_term_memory = '' #@param {type:\"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "12zM0VBzRy86"
      },
      "source": [
        "Answer:\n",
        "\n",
        "It is  impossible  to summarize any long document, like a  wiki page, in a single vector, so answering a question where the answer is somewhere in a wiki pages is too hard for  a seq2seq  model.  \n",
        "\n",
        "Most long  term memory storage in computers is done in databases (SQL or noSQL), so  one  might want to make neural lets that  read from or write to databases or, more generally, files."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G0zmwYBImHdn"
      },
      "source": [
        "---\n",
        "# Section 2: Time series with neural nets\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gMyTYiaky5aQ",
        "cellView": "form"
      },
      "source": [
        "#@title Video: Time Series\n",
        "try: t2;\n",
        "except NameError: t2=time.time()\n",
        "\n",
        "video = YouTubeVideo(id=\"nVmD3uwGnTc\", width=854, height=480, fs=1)\n",
        "print(\"Video available at https://youtube.com/watch?v=\" + video.id)\n",
        "\n",
        "video\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W3iT9yfD7xCo"
      },
      "source": [
        "Most time series modeling mostly uses recurrent neural nets. Why?\n",
        "There is an active line of research trying to replace RNNs with methods that use padding+truncation.  Why?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5LjkeJK18KHS",
        "cellView": "form"
      },
      "source": [
        "why_RNNs = '' #@param {type:\"string\"}\n",
        "why_not_RNNs = '' #@param {type:\"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FBdqgsJdSNTQ"
      },
      "source": [
        "Answer:\n",
        "\n",
        "RNNs are attractive for time series  because  they can handle sequences of  arbitrary length.  \n",
        "\n",
        "However,  it is faster to do matrix operations on vectors of fixed length, and minibatch gradient descent is easier to implement efficiently if you know exactly how long your vectors are."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6JkNScTAc5qs"
      },
      "source": [
        "*Estimated time: 40 minutes since start*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P5dk1rRCXlrI"
      },
      "source": [
        "---\n",
        "# Section 3: Recurrence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jow_fm5rzdbv",
        "cellView": "form"
      },
      "source": [
        "#@title Video: Recurrence\n",
        "try: t3;\n",
        "except NameError: t3=time.time()\n",
        "\n",
        "video = YouTubeVideo(id=\"XDdt2qN7PtY\", width=854, height=480, fs=1)\n",
        "print(\"Video available at https://youtube.com/watch?v=\" + video.id)\n",
        "\n",
        "video"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l3lMLaEl8RRt"
      },
      "source": [
        "A key assumption behind RNNs is stationarity. Consider the problem of forecasting daily sales of different products (e.g. for amazon). Is that time series stationary? What might you do (e.g., in terms of additional features) to make it more stationary?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W8Jv6WBg8y9k",
        "cellView": "form"
      },
      "source": [
        "creating_stationarity = '' #@param {type:\"string\"}\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nCJxnryESWKw"
      },
      "source": [
        "Answer:\n",
        "\n",
        "Product sales by, say day, is not a  stationary series. There is “seasonality. There are variations as a function of the day of the week, whether or not it is a holiday,, how close it is to Christmas  or another holiday,  etc.  These can be addressed by making all of those attributes of the day be features that are  also fed into the neural net."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f6gHudTqTuyd"
      },
      "source": [
        "---\n",
        "# Section 4: Embeddings\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YZ0k78G2zolq",
        "cellView": "form"
      },
      "source": [
        "#@title Video: Embeddings\n",
        "try: t4;\n",
        "except NameError: t4=time.time()\n",
        "\n",
        "video = YouTubeVideo(id=\"LcTAsly3yhI\", width=854, height=480, fs=1)\n",
        "print(\"Video available at https://youtube.com/watch?v=\" + video.id)\n",
        "\n",
        "video"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GDk5_9UobCfo"
      },
      "source": [
        "[Word2vec](https://rare-technologies.com/word2vec-tutorial/) is a group of related models that are used to produce word embeddings. These models are shallow, two-layer neural networks that are trained to reconstruct linguistic contexts of words. Word2vec takes as its input a large corpus of text and produces a vector space, with each unique word in the corpus being assigned a corresponding vector in the space. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4hYXzuYe54uP"
      },
      "source": [
        "## Section 4.1: Creating Word Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qn_MEZO6fLaa"
      },
      "source": [
        "We will create embeddings for a subset of categories in [Brown corpus](https://www1.essex.ac.uk/linguistics/external/clmt/w3c/corpus_ling/content/corpora/list/private/brown/brown.html).  In order to achieve this task we will use [gensim](https://radimrehurek.com/gensim/) library to create word2vec embeddings. Gensim’s word2vec expects a sequence of sentences as its input. Each sentence is a list of words.\n",
        "Calling Word2Vec(sentences, iter=1) will run two passes over the sentences iterator (or, in general iter+1 passes). The first pass collects words and their frequencies to build an internal dictionary tree structure. The second and subsequent passes train the neural model. \n",
        "Word2vec accepts several parameters that affect both training speed and quality.\n",
        "\n",
        "One of them is for pruning the internal dictionary. Words that appear only once or twice in a billion-word corpus are probably uninteresting typos and garbage. In addition, there’s not enough data to make any meaningful training on those words, so it’s best to ignore them:\n",
        "\n",
        "`model = Word2Vec(sentences, min_count=10)  # default value is 5`\n",
        "\n",
        "\n",
        "A reasonable value for min_count is between 0-100, depending on the size of your dataset.\n",
        "\n",
        "Another parameter is the size of the NN layers, which correspond to the “degrees” of freedom the training algorithm has:\n",
        "\n",
        "`model = Word2Vec(sentences, size=200)  # default value is 100`\n",
        "\n",
        "\n",
        "Bigger size values require more training data, but can lead to better (more accurate) models. Reasonable values are in the tens to hundreds.\n",
        "\n",
        "The last of the major parameters (full list [here](https://radimrehurek.com/gensim/models/word2vec.html#gensim.models.word2vec.Word2Vec)) is for training parallelization, to speed up training:\n",
        "\n",
        "`model = Word2Vec(sentences, workers=4) # default = 1 worker = no parallelization`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4UhrgWC9yNw-"
      },
      "source": [
        "category = ['editorial', 'fiction', 'government', 'mystery', 'news', 'religion',\n",
        "            'reviews', 'romance', 'science_fiction']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mYG6dRgl4znS"
      },
      "source": [
        "def create_word2vec_model(category = 'news', size = 50, sg = 1, min_count = 5):\n",
        "  try:\n",
        "    sentences = brown.sents(categories=brown.categories()) \n",
        "    model = Word2Vec(sentences, size=size, sg=sg, min_count=min_count)\n",
        "\n",
        "  except (AttributeError, TypeError):\n",
        "      raise AssertionError('Input variable \"category\" should be a string or list,' \n",
        "      '\"size\", \"sg\", \"min_count\" should be integers')\n",
        "\n",
        "  return model\n",
        "\n",
        "def model_dictionary(model):\n",
        "  words = list(w2vmodel.wv.vocab)\n",
        "  return words \n",
        "\n",
        "def get_embedding(word, model):\n",
        "  if word in w2vmodel.wv.vocab:\n",
        "    return model.wv[word]\n",
        "  else:\n",
        "    return None"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E5xulV57Nw2o"
      },
      "source": [
        "The cell will take 30-45 seconds to run."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P0LQa50W1d3v"
      },
      "source": [
        "w2vmodel = create_word2vec_model(category)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KQj91u9L3ZEz"
      },
      "source": [
        "You can list all the words for which embeddings are created i.e. the dictionary."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5lwU0J0SUXRs"
      },
      "source": [
        "print(model_dictionary(w2vmodel))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d7WY_g_P3wXY"
      },
      "source": [
        "You can get the embedding vector for a word in the dictionary."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Va12snm4Itt"
      },
      "source": [
        "print(get_embedding('weather', w2vmodel))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CO1xq4G26C2e"
      },
      "source": [
        "What do the embeddings mean? What do they represent?\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "KfO30iIf2Uum"
      },
      "source": [
        "word_embedding_meaning = '' #@param {type:\"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YITWgmtw59s9"
      },
      "source": [
        "## Section 4.2: Visualizing Word Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wTrSH9LKP5hM"
      },
      "source": [
        "We can now obtain the word embeddings for any word in the dictionary using word2vec. Let's visualize these embeddings to get an inuition of what these embeddings mean. The word embeddings obtained from word2vec model are in high dimensional space. We will use PCA to pick the 2 features that capture the most variance in the embeddings in order to represent them in a 2D space.\n",
        "\n",
        "\n",
        "For each word in `keys`, we pick the top 10 similar words (using cosine similarity) and plot them.  \n",
        "\n",
        " What should be the arrangement of similar words?\n",
        " What should be arrangement of the key clusters with respect to each other?\n",
        " "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PIxN4T_xRLEC"
      },
      "source": [
        "keys = ['voters', 'magic', 'love', 'God', 'evidence', 'administration', 'governments']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y1QP-C3WJYLl"
      },
      "source": [
        "def get_cluster_embeddings(keys):\n",
        "  embedding_clusters = []\n",
        "  word_clusters = []\n",
        "\n",
        "  # find closest words and add them to cluster\n",
        "  for word in keys:\n",
        "      embeddings = []\n",
        "      words = []\n",
        "      if not word in w2vmodel.wv.vocab:\n",
        "        print('The word ', word, 'is not in the dictionary')\n",
        "        continue\n",
        "\n",
        "      for similar_word, _ in w2vmodel.wv.most_similar(word, topn=10):\n",
        "          words.append(similar_word)\n",
        "          embeddings.append(w2vmodel.wv[similar_word])\n",
        "      embedding_clusters.append(embeddings)\n",
        "      word_clusters.append(words)\n",
        "\n",
        "  # get embeddings for the words in clusers\n",
        "  embedding_clusters = np.array(embedding_clusters)\n",
        "  n, m, k = embedding_clusters.shape\n",
        "  tsne_model_en_2d = TSNE(perplexity=10, n_components=2, init='pca', n_iter=3500, random_state=32)\n",
        "  embeddings_en_2d = np.array(tsne_model_en_2d.fit_transform(embedding_clusters.reshape(n * m, k))).reshape(n, m, 2)\n",
        "  return embeddings_en_2d, word_clusters"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2M9TuaqJKUbo"
      },
      "source": [
        "def tsne_plot_similar_words(title, labels, embedding_clusters, word_clusters, a, filename=None):\n",
        "    plt.figure(figsize=(16, 9))\n",
        "    colors = cm.rainbow(np.linspace(0, 1, len(labels)))\n",
        "    for label, embeddings, words, color in zip(labels, embedding_clusters, word_clusters, colors):\n",
        "        x = embeddings[:, 0]\n",
        "        y = embeddings[:, 1]\n",
        "        plt.scatter(x, y, color=color, alpha=a, label=label)\n",
        "        for i, word in enumerate(words):\n",
        "            plt.annotate(word, alpha=0.5, xy=(x[i], y[i]), xytext=(5, 2),\n",
        "                         textcoords='offset points', ha='right', va='bottom', size=10)\n",
        "    plt.legend(loc=\"lower left\")\n",
        "    plt.title(title)\n",
        "    plt.grid(True)\n",
        "    if filename:\n",
        "        plt.savefig(filename, format='png', dpi=150, bbox_inches='tight')\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJvOa5X1SeXv"
      },
      "source": [
        "embeddings_en_2d, word_clusters = get_cluster_embeddings(keys)\n",
        "tsne_plot_similar_words('Similar words from Brown Corpus', keys, embeddings_en_2d, word_clusters, 0.7,\n",
        "                        'similar_words.png')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JbMNDejeTvgr"
      },
      "source": [
        "Analyze the plot and answer the questions below - \n",
        "\n",
        "\n",
        "What does having higher similarity between two word embeddings mean?\n",
        "\n",
        "\n",
        "\n",
        "Why are cluster centers close to some keys but farther from others?\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Cjp2U7F7pva",
        "cellView": "form"
      },
      "source": [
        "higher_similarity_meaning = '' #@param {type:\"string\"}\n",
        "cluster_centers = '' #@param {type:\"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y3efrPxtdHZV"
      },
      "source": [
        "*Estimated time: 70 minutes since start*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ro22isScXgdn"
      },
      "source": [
        "---\n",
        "# Section 5: Recurrent Neural Networks (RNNs)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TXXTYr460qdv",
        "cellView": "form"
      },
      "source": [
        "#@title Video: RNN\n",
        "try: t5;\n",
        "except NameError: t5=time.time()\n",
        "\n",
        "video = YouTubeVideo(id=\"xjAbQQFpojo\", width=854, height=480, fs=1)\n",
        "print(\"Video available at https://youtube.com/watch?v=\" + video.id)\n",
        "\n",
        "video"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nORAQP36wy_L"
      },
      "source": [
        "## Section 5.1: Introduction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ip_QWG1So9Hl"
      },
      "source": [
        "Recurrent neural networks, or RNNs , are a family of neural networks for processing sequential data. Just as a convolutional network is specialized for processing a grid of values X such as an image, a recurrent neural network is specialized for processing a sequence of values. RNNs prove useful in many scenarios where other deep learning models are not effective.\n",
        "\n",
        "* Not all problems can be converted into one with fixed length inputs and outputs.\n",
        "\n",
        "\n",
        "* The deep learning models we have seen so far pick samples randomly. This might not be the best strategy for a task of understanding meaning from a piece of text. Words in a text occur in a sequence and therefore cannot be permuted randomly to get the meaning. \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vJgOfSg4sTpr"
      },
      "source": [
        "A related idea is the use of convolution across a 1-D temporal sequence. The convolution operation allows a network to share parameters across time but is shallow. The output of convolution is a sequence where each output is a function of a small number of neighboring inputs. The idea of parameter sharing manifests in the application of the same convolution kernel at each time step. Recurrent networks share parameters in a diﬀerent way. Each output is a function of the previous hidden layer, always produced using the same model, with the same weights. This recurrent formulation results in the sharing of parameters through a very deep computational graph.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eue9vx0w3Ll"
      },
      "source": [
        "## Section 5.2: Working of a RNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2WjBtwjstNH1"
      },
      "source": [
        "The following provides more data than the video (but can be skipped for now). For more detail, see the sources, the [deep learning book](https://www.deeplearningbook.org/contents/rnn.html), and [d2l.ai](https://d2l.ai/chapter_recurrent-neural-networks/rnn.html)\n",
        "\n",
        "When the recurrent network is trained to perform a task that requires predicting the future from the past, the network typically learns to use a hidden state at time step $t$, $H_t$ as a kind of lossy summary of the task-relevant aspects of the past sequence of inputs up to $t$. This summary is in general necessarily lossy, since it maps an arbitrary length sequence $(X_t, X_{t-1}, X_{t-2}, . . . , X_{2}, X_{1})$ to a ﬁxed length vector $H_t$.\n",
        "\n",
        "\n",
        "\n",
        "We can represent the unfolded recurrence after $t$ steps with a function $G_t$: $$H_t=G_t(X_t, X_{t-1}, X_{t-2}, . . . , X_{2}, X_{1})$$ $$=f(H_{t−1}, X_{t}; \\theta)$$\n",
        "\n",
        "\n",
        "<img width=700 src=\"https://blog.floydhub.com/content/images/2019/04/rnn-2.gif\"> \n",
        "\n",
        "The function $g_t$ takes the whole past sequence $(X_t, X_{t-1}, X_{t-2}, . . . , X_{2}, X_{1})$ as input and produces the current state, but the unfolded recurrent structure allows us to factorize $g_t$ into repeated application of a function f. The unfolding process thus introduces two major advantages-\n",
        "\n",
        "* Regardless of the sequence length, the learned model always has the same input size, because it is speciﬁed in terms of transition from one state to another state, rather than speciﬁed in terms of a variable-length history of states.\n",
        "\n",
        "* It is possible to use the same transition function $f$ with the same parameters at every time step.\n",
        "\n",
        "\n",
        "We will now formally write down the equations of a recurrent unit-\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "Assume that we have a minibatch of inputs $X_t \\in R^{nxd}$ at time step $t$ . In other words, for a minibatch of $n$ sequence examples, each row of $X_t$  corresponds to one example at time step $t$ from the sequence. Next, we denote by $H_t \\in R^{nxh}$ the hidden variable of time step $t$. Unlike the MLP, here we save the hidden variable $H_{t-1}$ from the previous time step and introduce a new weight parameter $W_{hh} \\in R^{hxh}$ to describe how to use the hidden variable of the previous time step in the current time step. Specifically, the calculation of the hidden variable of the current time step is determined by the input of the current time step together with the hidden variable of the previous time step:\n",
        "\n",
        "$$H_t = \\phi(X_t W_{xh} + H_{t-1}W_{hh} + b_h)$$\n",
        "\n",
        "For time step $t$, the output of the output layer is similar to the computation in the MLP:\n",
        "\n",
        "$$O_t = H_t W_{hq} + b_q$$\n",
        " \n",
        "Parameters of the RNN include the weights $W_{xh} \\in R^{dxh}, W_{hh} \\in R^{hxh}$ , and the bias $b_h \\in R^{1xh}$ of the hidden layer, together with the weights $W_{hq} \\in R^{hxq}$ and the bias $b_q \\in R^{1xq}$ of the output layer. It is worth mentioning that even at different time steps, RNNs always use these model parameters. Therefore, the parameterization cost of an RNN does not grow as the number of time steps increases.\n",
        "\n",
        "<img width=700 align=\"center\" src=\"https://d2l.ai/_images/rnn.svg\">"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uu6juESc4_8H"
      },
      "source": [
        "## Section 5.3: Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Zyj7pMM5Cqq"
      },
      "source": [
        "We will use the IMDB dataset from <insert link>, which consists of a set of 25,000 highly polar movie reviews for training, and 25,000 for testing. We will use torchtext to download the dataset and prepare it for training, validation and testing. Our goal is to build a model that performs binary classification between positive and negative movie reviews.\n",
        "\n",
        "We use `fix_length` argument to pad sentences of length less than `sentence_length` or truncate sentences of length greater than `sentence_length`. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YIx3PXMlP6WE"
      },
      "source": [
        "batch_size = 32 \n",
        "\n",
        "def load_dataset(sentence_length = 50):\n",
        "    TEXT = data.Field(sequential=True, tokenize=nltk.word_tokenize, lower=True,\n",
        "                      include_lengths=True, batch_first=True, fix_length=sentence_length)\n",
        "    LABEL = data.LabelField(dtype=torch.float)\n",
        "\n",
        "    train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)\n",
        "\n",
        "    # If no specific vector embeddings are specified,\n",
        "    # Torchtext initializes random vector embeddings\n",
        "    # which would get updated during training through backpropagation.\n",
        "    TEXT.build_vocab(train_data)\n",
        "    LABEL.build_vocab(train_data)\n",
        "\n",
        "    train_data, valid_data = train_data.split(split_ratio=0.7, random_state = random.seed(seed))\n",
        "    train_iter, valid_iter, test_iter = data.BucketIterator.splits((train_data, valid_data, test_data),\n",
        "                                                                   batch_size=batch_size, sort_key=lambda x: len(x.text),\n",
        "                                                                   repeat=False, shuffle=True)\n",
        "    vocab_size = len(TEXT.vocab)\n",
        "\n",
        "    return TEXT, vocab_size, train_iter, valid_iter, test_iter"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WY3XvpFDYWpd"
      },
      "source": [
        "The cell below can take 15-30 secs to run."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LMpmMB2cbU3r"
      },
      "source": [
        "TEXT, vocab_size, train_iter, valid_iter, test_iter = load_dataset()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xDe_B8VF74wm"
      },
      "source": [
        "Let's see what the data looks like. The words in the reviews are tokenized using the NLTK `word_tokenize` function.  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kPikFvHS7mHb"
      },
      "source": [
        "def text_from_dict(arr, dictionary):\n",
        "  text = []\n",
        "  for element in arr:\n",
        "    text.append(dictionary[element])\n",
        "  return text\n",
        "\n",
        "for idx, batch in enumerate(train_iter):\n",
        "    text = batch.text[0]\n",
        "    target = batch.label\n",
        "\n",
        "    for itr in range(25,30):\n",
        "      print('Review: ', ' '.join(text_from_dict(text[itr], TEXT.vocab.itos)))\n",
        "      print('Label: ', int(target[itr].item()), '\\n')\n",
        "   \n",
        "    print('[0: Negative Review, 1: Positive Review]')\n",
        "    if idx==0:\n",
        "      break"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dt_sTnkYcQrK",
        "cellView": "form"
      },
      "source": [
        "#@title Training\n",
        "def train(model, device, train_iter, valid_iter, epochs, learning_rate):\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "    \n",
        "    train_loss, validation_loss = [], []\n",
        "    train_acc, validation_acc = [], []\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "      #train\n",
        "      model.train()\n",
        "      running_loss = 0.\n",
        "      correct, total = 0, 0 \n",
        "      steps = 0\n",
        "\n",
        "      for idx, batch in enumerate(train_iter):\n",
        "        text = batch.text[0]\n",
        "        # print(type(text), text.shape)\n",
        "        target = batch.label\n",
        "        target = torch.autograd.Variable(target).long()\n",
        "        text, target = text.to(device), target.to(device)\n",
        "\n",
        "        # add micro for coding training loop\n",
        "        optimizer.zero_grad()\n",
        "        output = model(text)\n",
        " \n",
        "        loss = criterion(output, target)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        steps += 1\n",
        "        running_loss += loss.item()\n",
        "\n",
        "        # get accuracy \n",
        "        _, predicted = torch.max(output, 1)\n",
        "        total += target.size(0)\n",
        "        correct += (predicted == target).sum().item()\n",
        "        \n",
        "      train_loss.append(running_loss/len(train_iter))\n",
        "      train_acc.append(correct/total)\n",
        "\n",
        "      print(f'Epoch: {epoch + 1},  Training Loss: {running_loss/len(train_iter):.4f}, Training Accuracy: {100*correct/total: .2f}%')\n",
        "\n",
        "      # evaluate on validation data\n",
        "      model.eval()\n",
        "      running_loss = 0.\n",
        "      correct, total = 0, 0 \n",
        "\n",
        "      with torch.no_grad():\n",
        "        for idx, batch in enumerate(valid_iter):\n",
        "            text = batch.text[0]\n",
        "            target = batch.label\n",
        "            target = torch.autograd.Variable(target).long()\n",
        "            text, target = text.to(device), target.to(device)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            output = model(text)\n",
        "    \n",
        "            loss = criterion(output, target)\n",
        "            running_loss += loss.item()\n",
        "\n",
        "            # get accuracy \n",
        "            _, predicted = torch.max(output, 1)\n",
        "            total += target.size(0)\n",
        "            correct += (predicted == target).sum().item()\n",
        "\n",
        "      validation_loss.append(running_loss/len(valid_iter))\n",
        "      validation_acc.append(correct/total)\n",
        "\n",
        "      print (f'Validation Loss: {running_loss/len(valid_iter):.4f}, Validation Accuracy: {100*correct/total: .2f}%')\n",
        "  \n",
        "    return train_loss, train_acc, validation_loss, validation_acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NCz88uW3bBW7",
        "cellView": "form"
      },
      "source": [
        "#@title Testing\n",
        "def test(model,  device, test_iter):\n",
        "  model.eval()\n",
        "  correct = 0\n",
        "  total = 0\n",
        "  with torch.no_grad():\n",
        "    for idx, batch in enumerate(test_iter):\n",
        "        text = batch.text[0]\n",
        "        target = batch.label\n",
        "        target = torch.autograd.Variable(target).long()\n",
        "        text, target = text.to(device), target.to(device)\n",
        "\n",
        "        outputs = model(text)\n",
        "        _, predicted = torch.max(outputs, 1)\n",
        "        total += target.size(0)\n",
        "        correct += (predicted == target).sum().item()\n",
        "\n",
        "    acc = 100 * correct / total\n",
        "    return acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TF3TDTI8qqTG",
        "cellView": "form"
      },
      "source": [
        "#@title Helper Functions\n",
        "# helpers\n",
        "def plot_train_val(x, train, val, train_label, val_label, title):\n",
        "  plt.plot(x, train, label=train_label)\n",
        "  plt.plot(x, val, label=val_label)\n",
        "  plt.legend(loc='lower right')\n",
        "  plt.xlabel('epoch')\n",
        "  plt.title(title)\n",
        "  plt.show()\n",
        "\n",
        "def count_parameters(model):\n",
        "    parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "    return parameters\n",
        "\n",
        "def init_weights(m):\n",
        "    if type(m) in (nn.Linear, nn.Conv1d):\n",
        "        nn.init.xavier_uniform_(m.weight)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kXnDD1STxPNv"
      },
      "source": [
        "## Section 5.4: 1D CNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eCKTZ5wnD4MX"
      },
      "source": [
        "Over the last few weeks, you were introduced to CNNs and used them to work on a range of interesting deep learning applications in vision. You also discussed where else these networks might be useful. Can we apply CNNs to language?\n",
        "\n",
        "Before we jump into RNNs we will create a CNN model for the text classification task. Let us understand how a one-dimensional convolutional layer works-\n",
        "\n",
        "<img width=700 src=https://d2l.ai/_images/conv1d.svg>\n",
        "\n",
        "The above figure shows one-dimensional cross-correlation operation. The shaded parts are the first output element as well as the input and kernel array elements used in its calculation:  0×1+1×2=2 .\n",
        "\n",
        "<img width=700 src=https://d2l.ai/_images/conv1d-channel.svg>\n",
        "\n",
        "The above figure shows one-dimensional cross-correlation operation with three input channels. The shaded parts are the first output element as well as the input and kernel array elements used in its calculation:  0×1+1×2+1×3+2×4+2×(−1)+3×(−3)=2 .\n",
        "\n",
        "Similarly, we have a one-dimensional pooling layer. The max-over-time pooling layer used in CNN actually corresponds to a one-dimensional global maximum pooling layer. Assuming that the input contains multiple channels, and each channel consists of values on different time steps, the output of each channel will be the largest value of all time steps in the channel. Therefore, the input of the max-over-time pooling layer can have different time steps on each channel.\n",
        "\n",
        "Fitting all the pieces together a 1D CNN architecture looks like the image below - \n",
        "\n",
        "<img width=700 src=https://d2l.ai/_images/textcnn.svg>\n",
        "\n",
        "In this figure, we have an input sequence of 11 words, each with 6 channels. Our model consists of two Conv1D layers applied to this input: one of kernel size 2 having 4 output channels and one of kernel size 4 having 5 output channels.  The max-over-time pooling is applied to each of the output channels. The max-pooled outputs for each output channel of the Conv1D layers are concatenated to form 1x4 and 1x5 vectors respectively. Finally these outputs are further concatenated to form a vector of 1x9 and this is passed to a fully connected layer of size 2 for binary classification. \n",
        "\n",
        "The intuition of using a max-pooling over different channels here is to capture the most important features over time in an input sequence.  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dooLZs-cXCl6"
      },
      "source": [
        "### Exercise 1\n",
        "\n",
        "Now it's your turn to implement a 1D CNN. \n",
        "\n",
        "* Here we will use [`nn.Embedding`](https://pytorch.org/docs/stable/generated/torch.nn.Embedding.html)  layer instead of pretrained word embeddings. This is a design choice. Using `nn.Embedding` layer in the network allows us to train word embeddings specific to the problem at hand. You are given the `vocab_size` which is the size of the dictionary of embeddings, and the `embed_size` which is the size of each embedding vector. \n",
        "\n",
        "* The 1D CNN should work for any number kernel size and corresponsing number of channels. Both `kernel_sizes` and `num_channels` are lists. Each element of these lists corresponds to the kernel size and output channels of a Conv1D layer.\n",
        "\n",
        "* Use [max pooling](https://pytorch.org/docs/stable/generated/torch.nn.AdaptiveMaxPool1d.html) \n",
        "\n",
        "* Determine the size of inputs and outputs to the fully-connected layer using the reference example given above.  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8hT-3pPnZLpI"
      },
      "source": [
        "class TextCNN(nn.Module):\n",
        "    def __init__(self, vocab_size, embed_size, kernel_sizes, num_channels,\n",
        "                 **kwargs):\n",
        "        super(TextCNN, self).__init__(**kwargs)\n",
        "\n",
        "        ####################################################################\n",
        "        # Fill in missing code below (...),\n",
        "        # then remove or comment the line below to test your function\n",
        "        raise NotImplementedError(\"TextCNN\")\n",
        "        ####################################################################\n",
        "\n",
        "        self.embedding = ...\n",
        "        self.fc = ...\n",
        "        self.pool = ...\n",
        "        self.relu = ...\n",
        "        self.convs = nn.ModuleList()\n",
        "        # This for loop adds the Conv1D layers to your network\n",
        "        for c, k in zip(..., ...):\n",
        "            self.convs.append(nn.Conv1d(..., ..., ...))\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        embeddings = self.embedding(inputs)\n",
        "        embeddings = embeddings.permute(0, 2, 1)\n",
        "        # Concatenating the average-pooled outputs  \n",
        "        encoding = torch.cat([\n",
        "            torch.squeeze(self.relu(self.pool(conv(embeddings))), dim=-1)\n",
        "            for conv in self.convs], dim=1)\n",
        "        outputs = self.fc(encoding)\n",
        "        return outputs\n",
        "\n",
        "# # Uncomment to test\n",
        "# sampleCNN = TextCNN(1000, 300, [1, 2, 3], [10, 20, 30])\n",
        "# print(sampleCNN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bj_Bv8f4NMGH"
      },
      "source": [
        "Sample output:\n",
        "\n",
        "```\n",
        "TextCNN(\n",
        "  (embedding): Embedding(1000, 300)\n",
        "  (fc): Linear(in_features=60, out_features=2, bias=True)\n",
        "  (pool): AdaptiveMaxPool1d(output_size=1)\n",
        "  (relu): ReLU()\n",
        "  (convs): ModuleList(\n",
        "    (0): Conv1d(300, 10, kernel_size=(1,), stride=(1,))\n",
        "    (1): Conv1d(300, 20, kernel_size=(2,), stride=(1,))\n",
        "    (2): Conv1d(300, 30, kernel_size=(3,), stride=(1,))\n",
        "  )\n",
        ")\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f_-_rCPXdZ2Q"
      },
      "source": [
        "# to_remove solution\n",
        "class TextCNN(nn.Module):\n",
        "    def __init__(self, vocab_size, embed_size, kernel_sizes, num_channels,\n",
        "                 **kwargs):\n",
        "        super(TextCNN, self).__init__(**kwargs)\n",
        "        self.embedding = nn.Embedding(vocab_size, embed_size)\n",
        "        self.fc = nn.Linear(sum(num_channels), 2)\n",
        "        self.pool = nn.AdaptiveMaxPool1d(1)\n",
        "        self.relu = nn.ReLU()\n",
        "        self.convs = nn.ModuleList()\n",
        "        # This for loop adds the Conv1D layers to your network\n",
        "        for c, k in zip(num_channels, kernel_sizes):\n",
        "            self.convs.append(nn.Conv1d(embed_size, c, k))\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        embeddings = self.embedding(inputs)\n",
        "        embeddings = embeddings.permute(0, 2, 1)\n",
        "        # Concatenating the average-pooled outputs \n",
        "        encoding = torch.cat([\n",
        "            torch.squeeze(self.relu(self.pool(conv(embeddings))), dim=-1)\n",
        "            for conv in self.convs], dim=1)\n",
        "        \n",
        "        outputs = self.fc(encoding)\n",
        "        return outputs\n",
        "\n",
        "# Uncomment to test\n",
        "sampleCNN = TextCNN(1000, 300, [1, 2, 3], [10, 20, 30])\n",
        "print(sampleCNN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r_ssg4QLbIJX"
      },
      "source": [
        "# Model hyperparameters\n",
        "learning_rate = 0.00001\n",
        "embedding_length = 100\n",
        "kernel_sizes = [3, 4, 5]\n",
        "nums_channels = [100, 100, 100]\n",
        "epochs = 10\n",
        "\n",
        "# Initialize model, training and testing\n",
        "cnn_model = TextCNN(vocab_size, embedding_length, kernel_sizes, nums_channels)\n",
        "cnn_model.to(device)\n",
        "cnn_model.apply(init_weights)\n",
        "cnn_start_time = time.time()\n",
        "cnn_train_loss, cnn_train_acc, cnn_validation_loss, cnn_validation_acc = train(cnn_model, device, train_iter, valid_iter, epochs, learning_rate)\n",
        "print(\"--- Time taken to train = %s seconds ---\" % (time.time() - cnn_start_time))\n",
        "test_accuracy = test(cnn_model, device, test_iter)\n",
        "print('Test Accuracy: ',  test_accuracy, '%')\n",
        "\n",
        "# Plot accuracies\n",
        "plot_train_val(np.arange(0,epochs), cnn_train_acc, cnn_validation_acc, 'training_accuracy', 'validation_accuracy', 'CNN on IMDB text classification')\n",
        "\n",
        "# Number of parameters in model\n",
        "paramters = count_parameters(cnn_model)\n",
        "print('\\n\\nNumber of parameters = ',  paramters)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2i6HuNNn5HLN"
      },
      "source": [
        "What features does 1D CNN learn?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "U7qJ4ous71VG"
      },
      "source": [
        "conv1D_features = '' #@param {type:\"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dGLergw_5_z7"
      },
      "source": [
        "to_remove solution\n",
        "\n",
        "These heights effectively capture patterns in sequential groups of 3, 4, and 5 words. A cutoff of 5 words because words that were farther away than that were generally less relevant or useful with respect to identifying patterns in a phrase."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3tn4t56cdhZF"
      },
      "source": [
        "*Estimated time: 95 minutes since start*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UvoEGyF9fahG"
      },
      "source": [
        "## Section 5.5: Vanilla RNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PID0RcvPr8h1"
      },
      "source": [
        "### Exercise 2\n",
        "\n",
        "Now it's your turn to write a Vanilla RNN using PyTorch.\n",
        "\n",
        "* Once again we will use `nn.Embedding`. You are given the `vocab_size` which is the size of the dictionary of embeddings, and the `embed_size` which is the size of each embedding vector.  \n",
        "\n",
        "* Add 2 [RNN](https://pytorch.org/docs/stable/generated/torch.nn.RNN.html) layers. This would mean stacking two RNNs together to form a stacked RNN, with the second RNN taking in outputs of the first RNN and computing the final results.\n",
        "\n",
        "* Determine the size of inputs and outputs to the fully-connected layer.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dfVMmqlra45X"
      },
      "source": [
        "class VanillaRNN(nn.Module):\n",
        "  def __init__(self, output_size, hidden_size, vocab_size, embed_size):\n",
        "    super(VanillaRNN, self).__init__()\n",
        "    ####################################################################\n",
        "    # Fill in missing code below (...),\n",
        "    # then remove or comment the line below to test your function\n",
        "    raise NotImplementedError(\"Vanilla RNN\")\n",
        "    ####################################################################\n",
        "\n",
        "    self.hidden_size = ...\n",
        "\n",
        "    self.embeddings = ...\n",
        "    self.rnn = ...\n",
        "    self.fc = ...\n",
        "\n",
        "  def forward(self, inputs):\n",
        "    input = self.embeddings(inputs)\n",
        "    input = input.permute(1, 0, 2)\n",
        "    h_0 =  Variable(torch.zeros(2, input.size()[1], self.hidden_size).to(device)) \n",
        "    output, h_n = self.rnn(input, h_0)\n",
        "    h_n = h_n.permute(1, 0, 2) \n",
        "    h_n = h_n.contiguous().view(h_n.size()[0], h_n.size()[1]*h_n.size()[2])\n",
        "    logits = self.fc(h_n)\n",
        "    \n",
        "    return logits\n",
        "\n",
        "# # Uncomment to test\n",
        "# sampleRNN = VanillaRNN(10, 50, 1000, 300)\n",
        "# print(sampleRNN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Elc-WkNLSGN"
      },
      "source": [
        "Sample output:\n",
        "\n",
        "```\n",
        "VanillaRNN(\n",
        "  (word_embeddings): Embedding(1000, 100)\n",
        "  (rnn): RNN(100, 100, num_layers=2)\n",
        "  (fc): Linear(in_features=200, out_features=2, bias=True)\n",
        ")\n",
        "```\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5C6Xu1Oa7K4Q"
      },
      "source": [
        "# to_remove solution\n",
        "class VanillaRNN(nn.Module):\n",
        "  def __init__(self, output_size, hidden_size, vocab_size, embed_size):\n",
        "    super(VanillaRNN, self).__init__()\n",
        "\n",
        "    self.hidden_size = hidden_size\n",
        "    \n",
        "    self.word_embeddings = nn.Embedding(vocab_size, embed_size)\n",
        "    self.rnn = nn.RNN(embed_size, hidden_size, num_layers=2)\n",
        "    self.fc = nn.Linear(2*hidden_size, output_size)\n",
        "\n",
        "  def forward(self, inputs):\n",
        "    input = self.word_embeddings(inputs)\n",
        "    input = input.permute(1, 0, 2)\n",
        "    h_0 =  Variable(torch.zeros(2, input.size()[1], self.hidden_size).to(device)) \n",
        "    output, h_n = self.rnn(input, h_0)\n",
        "    h_n = h_n.permute(1, 0, 2) \n",
        "    h_n = h_n.contiguous().view(h_n.size()[0], h_n.size()[1]*h_n.size()[2])\n",
        "    logits = self.fc(h_n)\n",
        "    \n",
        "    return logits\n",
        "\n",
        "# Uncomment to test\n",
        "sampleRNN = VanillaRNN(10, 50, 1000, 300)\n",
        "print(sampleRNN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fwluUfZK79Tr"
      },
      "source": [
        "# Model hyperparamters\n",
        "learning_rate = 0.0002\n",
        "output_size = 2\n",
        "hidden_size = 100 #50\n",
        "embedding_length = 100\n",
        "epochs = 10\n",
        "\n",
        "# Initialize model, training and testing\n",
        "vanilla_rnn_model = VanillaRNN(output_size, hidden_size, vocab_size, embedding_length)\n",
        "vanilla_rnn_model.to(device)\n",
        "vanilla_rnn_start_time = time.time()\n",
        "vanilla_train_loss, vanilla_train_acc, vanilla_validation_loss, vanilla_validation_acc = train(vanilla_rnn_model, device, train_iter, valid_iter, epochs, learning_rate)\n",
        "print(\"--- Time taken to train = %s seconds ---\" % (time.time() - vanilla_rnn_start_time))\n",
        "test_accuracy = test(vanilla_rnn_model, device, test_iter)\n",
        "print('Test Accuracy: ',  test_accuracy, '%')\n",
        "\n",
        "# Plot accuracy curves\n",
        "plot_train_val(np.arange(0,epochs), vanilla_train_acc, vanilla_validation_acc,\n",
        "               'training_accuracy', 'validation_accuracy', 'Vanilla RNN on IMDB text classification')\n",
        "\n",
        "# Number of model parameters\n",
        "paramters = count_parameters(vanilla_rnn_model)\n",
        "print('\\n\\nNumber of parameters = ',  paramters)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-eIRg2TVxVZX"
      },
      "source": [
        "\n",
        " If we use one hot encoding instead of embedding, how many parameters woud the RNN model have?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "4Y5nuAkV8TuG"
      },
      "source": [
        "one_hot_parameters = '' #@param {type:\"string\"} "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UDm-0aWX_FNu"
      },
      "source": [
        "Based on exercises above would you say CNNs are faster than RNNs? Why?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "qkighpHi_YAi"
      },
      "source": [
        "cnn_vs_rnn_speed = '' #@param {type:  \"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-kbPkY0M_p9A"
      },
      "source": [
        "to_remove solution\n",
        "\n",
        "Yes, for this exercise. For a general case the answer depends on your model. CNN and RNN are different architectures, used differently, usually for different purposes. You can't really replace one by another without changing other elements of the model to compare the performance.\n",
        "\n",
        "However, CNN's are faster by design, since the computations in CNN's can happen in parallel (same filter applied to multiple locations of the image at the same time), while RNN's need to be processed sequentially, since the subsequent steps depend on previous ones. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Qc8N2rs-hRm"
      },
      "source": [
        "We know that an RNN can handle arbitrary lengths of inputs. So why might it be a good idea to pad/truncate the text here to make them  all the same length?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-iWQlT-V-p-g",
        "cellView": "form"
      },
      "source": [
        "why_pad = '' #@param {type:\"string\"}\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y-ybEJzPHaYH"
      },
      "source": [
        "## Section 5.6: Input Length\n",
        "\n",
        "Now let's increase the `sentence_length` to see how RNN performs when long reviews are allowed.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rd7APxyvHwAs"
      },
      "source": [
        "TEXT_long, vocab_size_long, train_iter_long, valid_iter_long, test_iter_long = load_dataset(200)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L_so4eTtcye0"
      },
      "source": [
        "The cell below will take 2-3 minutes to run."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L2NIGjpRH6R6"
      },
      "source": [
        "# Model hyperparamters\n",
        "learning_rate = 0.0002\n",
        "output_size = 2\n",
        "hidden_size = 100\n",
        "embedding_length = 100\n",
        "epochs = 10\n",
        "\n",
        "# Initialize model, training, testing\n",
        "vanilla_rnn_model_long = VanillaRNN(output_size, hidden_size, vocab_size_long, embedding_length)\n",
        "vanilla_rnn_model_long.to(device)\n",
        "vanilla_rnn_start_time_long = time.time()\n",
        "vanilla_train_loss_long, vanilla_train_acc_long, vanilla_validation_loss_long, vanilla_validation_acc_long = train(vanilla_rnn_model_long, device, train_iter_long, valid_iter_long, epochs, learning_rate)\n",
        "print(\"--- Time taken to train = %s seconds ---\" % (time.time() - vanilla_rnn_start_time_long))\n",
        "test_accuracy = test(vanilla_rnn_model_long, device, test_iter_long)\n",
        "print('Test Accuracy: ',  test_accuracy, '%')\n",
        "\n",
        "# Number of parameters\n",
        "paramters = count_parameters(vanilla_rnn_model_long)\n",
        "print('Number of parameters = ',  paramters)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TGjxzEuuIi3C"
      },
      "source": [
        "# Compare accuracies of model trained on different sentence lengths\n",
        "\n",
        "x_ticks = np.arange(epochs)\n",
        "plt.plot(x_ticks, vanilla_train_acc, label='train accuracy,len=50')\n",
        "plt.plot(x_ticks, vanilla_train_acc_long, label='train accuracy,len=200')\n",
        "\n",
        "plt.plot(x_ticks, vanilla_validation_acc, label='validation accuracy,len=50')\n",
        "plt.plot(x_ticks, vanilla_validation_acc_long, label='validation accuracy,len=200')\n",
        "\n",
        "plt.legend(loc='upper left')\n",
        "plt.xlabel('epoch')\n",
        "plt.title(\"Training and Validation Accuracy for Sentence Lengths 50 and 200\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "svnsA2wpK0WU"
      },
      "source": [
        "Compare the performance of Vanilla RNN for the different sentence lenghts and answer the questions below.\n",
        "\n",
        "Compared to sentence length of 50,  when using sentence length 200-"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "ibcxBV2CLJ71"
      },
      "source": [
        "training_time_200 = 'Select' #@param [\"increases\", \"decreases\", \"remains same\"] {allow-input: true}\n",
        "training_accuracy_200 = 'Select' #@param [\"increases\", \"decreases\", \"remains same\"] {allow-input: true}\n",
        "test_accuracy_200 = 'Select' #@param [\"increases\", \"decreases\", \"remains same\"] {allow-input: true}\n",
        "no_of_parameters_200 = 'Select' #@param [\"increases\", \"decreases\", \"remains same\"] {allow-input: true}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z1DFjHJtd4N8"
      },
      "source": [
        "*Estimated time: 120 minutes since start*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oYgqtChAwJGr"
      },
      "source": [
        "## Section 5.7: Architectures "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zwN8pHhzKJ_D"
      },
      "source": [
        "RNN models are mostly used in the fields of natural language processing and speech recognition. Below are types of RNNs. Depending on which outputs we use, RNN can be used for variety of tasks. The text classification problem we solved was an instance of the many to one architecture. Write down the applications of other architectures. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UbkMyG96I4VB"
      },
      "source": [
        "<img width=900 src=\"https://blog.floydhub.com/content/images/2019/04/karpathy.jpeg\">"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "y21sVcAP9a-k"
      },
      "source": [
        "one_to_one = '' #@param {type:\"string\"}\n",
        "\n",
        "one_to_many = '' #@param {type:\"string\"}\n",
        "\n",
        "many_to_one = '' #@param {type:\"string\"}\n",
        "\n",
        "many_to_many1 = '' #@param {type:\"string\"}\n",
        "\n",
        "many_to_many2 = '' #@param {type:\"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NVzzF_VmKquQ"
      },
      "source": [
        "to_remove solution\n",
        "\n",
        "\n",
        "Answer: \n",
        "* One to one - Traditional neural network\n",
        "* One to many - music generation\n",
        "* Many to one - sentiment classification/text classification\n",
        "* Many to many - machine translation\n",
        "* Many to many - named entity recognition/part-of-speech tagging\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1UUH2KjZw_IX"
      },
      "source": [
        "---\n",
        "# Section 6: Vanishing and Exploding Gradients\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pfGNDLFm2AsE",
        "cellView": "form"
      },
      "source": [
        "#@title Video: Backpropagation in RNNs\n",
        "try: t6;\n",
        "except NameError: t6=time.time()\n",
        "\n",
        "video = YouTubeVideo(id=\"O5fB4I50FpU\", width=854, height=480, fs=1)\n",
        "print(\"Video available at https://youtube.com/watch?v=\" + video.id)\n",
        "\n",
        "video"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3SXvP5FbhxEC"
      },
      "source": [
        "Problems with RNNs:\n",
        "\n",
        "* Vanishing gradients-\n",
        "In a long sequence, the gradients get multiplied by the weight matrix (transpose) at every time step. If there are small values in the weight matrix, the norm of gradients get smaller and smaller exponentially.\n",
        "* Exploding gradients-\n",
        "If we have a large weight matrix and the non-linearity in the recurrent layer is not saturating, the gradients will explode. The weights will diverge at the update step. We may have to use a tiny learning rate for the gradient descent to work.\n",
        "One reason to use RNNs is for the advantage of remembering information in the past. However, it could fail to memorize the information long ago in a simple RNN without tricks.\n",
        "\n",
        "An example that has vanishing gradient problem:\n",
        "\n",
        "The input is the characters from a C Program. The system will tell whether it is a syntactically correct program. A syntactically correct program should have a valid number of braces and parentheses. Thus, the network should remember how many open parentheses and braces there are to check, and whether we have closed them all. The network has to store such information in hidden states like a counter. However, because of vanishing gradients, it will fail to preserve such information in a long program."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YrD1vFThJm_2"
      },
      "source": [
        "Can you think of some other disadvantages of vanilla RNN?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "w2BVrKvUej1i"
      },
      "source": [
        "rnn_disadvantage = '' #@param {type: \"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yoQb_fDHJvQ4"
      },
      "source": [
        "to_remove solution\n",
        "\n",
        "\n",
        "Answer: Vanilla RNNs have the following disadvantages-\n",
        "* Computation being slow\n",
        "* Difficulty of accessing information from a long time ago\n",
        "* Cannot consider any future input for the current state"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qb4LyxqR1Sc1"
      },
      "source": [
        "## Section 6.1: Gradient Clipping\n",
        "\n",
        "While analyzing sequences through RNNs, we compute the gradients over a number of time steps iteratively, which results in a chain of matrix-products with length during backpropagation. This is likely to result in numerical instability; the gradients may either explode or vanish. To avoid these situations, RNN models often need extra help to stabilize the training.\n",
        "\n",
        "Sometimes the gradients can be quite large and the optimization algorithm may fail to converge. We could address this by reducing the learning rate. But what if we only rarely get large gradients? \n",
        "\n",
        "One popular alternative is to clip the gradient $g$  by projecting them back to a ball of a given radius, say $\\theta$ via\n",
        "\n",
        "$$g := min \\left[1, \\frac{\\theta}{∥g∥}\\right] . g $$\n",
        "\n",
        "The advantages of gradient clipping: \n",
        "- The gradient norm never exceeds the value $\\theta$ and that the updated gradient is entirely aligned with the original direction of $g$. \n",
        "- The influence of any given minibatch (and within it any given sample) exerted on the parameter vector is limited; which provides the model with robustness. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ukgspefqruEC"
      },
      "source": [
        "### Exercise 3\n",
        "Using the intuition above, write a function to perform gradient clipping, given a network and the value of $\\theta$.  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZIwZ77CVDmj9"
      },
      "source": [
        "def grad_clipping(net, theta):  \n",
        "    \"\"\"Clip the gradient.\"\"\"\n",
        "    ####################################################################\n",
        "    # Fill in missing code below (...),\n",
        "    # then remove or comment the line below to test your function\n",
        "    raise NotImplementedError(\"Gradient Clipping\")\n",
        "    ####################################################################\n",
        "\n",
        "    # Get a list of model parameters if they track gradients \n",
        "    params = ...\n",
        "    \n",
        "    # Compute the L2 norm of all the model's gradients\n",
        "    norm = ...\n",
        "\n",
        "    if norm > theta:\n",
        "        for param in params:\n",
        "            # Clip the gradient\n",
        "            param.grad[:] = ...\n",
        "\n",
        "# # Uncomment to test\n",
        "# sampleRNN2 = VanillaRNN(10, 50, 1000, 300).to(device)\n",
        "# x = torch.tensor([1, 2, 3]).view(1, -1).to(device)\n",
        "# output = sampleRNN2(x)\n",
        "# output.sum().backward()\n",
        "# grad_clipping(sampleRNN2, 10)\n",
        "# for p in sampleRNN2.parameters():\n",
        "#     assert torch.all(p.grad<=10)\n",
        "# print('Success!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iY5IW4AXsh_-"
      },
      "source": [
        "# to_remove solution\n",
        "def grad_clipping(net, theta):  \n",
        "    \"\"\"Clip the gradient.\"\"\"\n",
        "    params = [p for p in net.parameters() if p.requires_grad]\n",
        "\n",
        "    norm = torch.sqrt(sum(torch.sum((p.grad ** 2)) for p in params))\n",
        "    \n",
        "    if norm > theta:\n",
        "        for param in params:\n",
        "            param.grad[:] *= theta / norm\n",
        "    print(norm)\n",
        "\n",
        "# Uncomment to test\n",
        "sampleRNN2 = VanillaRNN(10, 50, 1000, 300).to(device)\n",
        "x = torch.tensor([1, 2, 3]).view(1, -1).to(device)\n",
        "output = sampleRNN2(x)\n",
        "output.sum().backward()\n",
        "grad_clipping(sampleRNN2, 10)\n",
        "for p in sampleRNN2.parameters():\n",
        "    assert torch.all(p.grad<=10)\n",
        "print('Success!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xARTXcOt0ia1"
      },
      "source": [
        "\n",
        "Besides gradient clipping, can you think of any other methods to cope with gradient explosion in recurrent neural networks?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "eDEbgPs-ep1s"
      },
      "source": [
        "grad_clipping_alternative = '' #@param {type: \"string\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UsVfoRwBDQ8z"
      },
      "source": [
        "to_remove solution\n",
        "\n",
        "Using specialized RNNs having \"gates\" that are better at retaining long term dependencies. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5YFx72OmePLu"
      },
      "source": [
        "*Estimated time: 130 minutes since start*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gbgTqgB5rH_W"
      },
      "source": [
        "---\n",
        "# Wrap up\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "98YXpSntrY6k",
        "cellView": "form"
      },
      "source": [
        "#@markdown #Run Cell to Show Airtable Form\n",
        "#@markdown ##**Confirm your answers and then click \"Submit\"**\n",
        "\n",
        "import time\n",
        "import numpy as np\n",
        "import urllib.parse\n",
        "from IPython.display import IFrame\n",
        "def prefill_form(src, fields: dict):\n",
        "  '''\n",
        "  src: the original src url to embed the form\n",
        "  fields: a dictionary of field:value pairs,\n",
        "  e.g. {\"pennkey\": my_pennkey, \"location\": my_location}\n",
        "  '''\n",
        "  prefill_fields = {}\n",
        "  for key in fields:\n",
        "      new_key = 'prefill_' + key\n",
        "      prefill_fields[new_key] = fields[key]\n",
        "  prefills = urllib.parse.urlencode(prefill_fields)\n",
        "  src = src + prefills\n",
        "  return src\n",
        "\n",
        "\n",
        "#autofill time if it is not present\n",
        "try: t0;\n",
        "except NameError: t0 = time.time()\n",
        "try: t1;\n",
        "except NameError: t1 = time.time()\n",
        "try: t2;\n",
        "except NameError: t2 = time.time()\n",
        "try: t3;\n",
        "except NameError: t3 = time.time()\n",
        "try: t4;\n",
        "except NameError: t4 = time.time()\n",
        "try: t5;\n",
        "except NameError: t5 = time.time()\n",
        "try: t6;\n",
        "except NameError: t6 = time.time()\n",
        "try: t7;\n",
        "except NameError: t7 = time.time()\n",
        "\n",
        "# autofill fields if they are not present\n",
        "# a missing pennkey and pod will result in an Airtable warning\n",
        "# which is easily fixed user-side.\n",
        "try: my_pennkey;\n",
        "except NameError: my_pennkey = \"\"\n",
        "try: my_pod;\n",
        "except NameError: my_pod = \"Select\"\n",
        "try: learning_from_previous_week;\n",
        "except NameError: learning_from_previous_week = \"\" \n",
        "try: where_RNNs_fail;\n",
        "except NameError: where_RNNs_fail = \"\"\n",
        "try: alternate_long_term_memory;\n",
        "except NameError: alternate_long_term_memory = \"\"\n",
        "try: why_RNNs;\n",
        "except NameError: why_RNNs = \"\"\n",
        "try: why_not_RNNs;\n",
        "except NameError: why_not_RNNs = \"\"\n",
        "try: creating_stationarity;\n",
        "except NameError: creating_stationarity = \"\"\n",
        "try: word_embedding_meaning;\n",
        "except NameError: word_embedding_meaning = \"\"\n",
        "try: higher_similarity_meaning;\n",
        "except NameError: higher_similarity_meaning = \"\"\n",
        "try: cluster_centers;\n",
        "except NameError: cluster_centers = \"\"\n",
        "try: conv1D_features;\n",
        "except NameError: conv1D_features = \"\"\n",
        "try: one_hot_parameters;\n",
        "except NameError: one_hot_parameters = \"\"\n",
        "try: cnn_vs_rnn_speed;\n",
        "except NameError: cnn_vs_rnn_speed = \"\"\n",
        "try: why_pad;\n",
        "except NameError: why_pad = \"\"\n",
        "try: training_time_200;\n",
        "except NameError: training_time_200 = \"\"\n",
        "try: training_accuracy_200;\n",
        "except NameError: training_accuracy_200 = \"\"\n",
        "try: test_accuracy_200;\n",
        "except NameError: test_accuracy_200 = \"\"\n",
        "try: no_of_parameters_200;\n",
        "except NameError: no_of_parameters_200 = \"\"\n",
        "try: one_to_one;\n",
        "except NameError: one_to_one = \"\"\n",
        "try: one_to_many;\n",
        "except NameError: one_to_many = \"\"\n",
        "try: many_to_one;\n",
        "except NameError: many_to_one = \"\"\n",
        "try: many_to_many1;\n",
        "except NameError: many_to_many1 = \"\"\n",
        "try: many_to_many2;\n",
        "except NameError: many_to_many2 = \"\"\n",
        "try: rnn_disadvantage;\n",
        "except NameError: rnn_disadvantage = \"\"\n",
        "try: grad_clipping_alternative;\n",
        "except NameError: grad_clipping_alternative = \"\"\n",
        "\n",
        "times = np.array([t1,t2,t3,t4,t5,t6,t7])-t0\n",
        "\n",
        "fields = {\n",
        "    \"my_pennkey\": my_pennkey,\n",
        "    \"my_pod\": my_pod, \n",
        "    \"learning_from_previous_week\": learning_from_previous_week,\n",
        "    \"word_embedding_meaning\": word_embedding_meaning,\n",
        "    \"higher_similarity_meaning\": higher_similarity_meaning,\n",
        "    \"cluster_centers\": cluster_centers,\n",
        "    \"conv1D_features\": conv1D_features,\n",
        "    \"one_hot_parameters\": one_hot_parameters,\n",
        "    \"cnn_vs_rnn_speed\": cnn_vs_rnn_speed,\n",
        "    \"training_time_200\": training_time_200,\n",
        "    \"training_accuracy_200\": training_accuracy_200,\n",
        "    \"test_accuracy_200\": test_accuracy_200,\n",
        "    \"no_of_parameters_200\": no_of_parameters_200,\n",
        "    \"one_to_one\": one_to_one,\n",
        "    \"one_to_many\": one_to_many,\n",
        "    \"many_to_one\": many_to_one,\n",
        "    \"many_to_many1\": many_to_many1,\n",
        "    \"many_to_many2\": many_to_many2,\n",
        "    \"rnn_disadvantage\": rnn_disadvantage,\n",
        "    \"grad_clipping_alternative\": grad_clipping_alternative,\n",
        "    \"where_RNNs_fail\": where_RNNs_fail,\n",
        "    \"alternate_long_term_memory\":alternate_long_term_memory,\n",
        "    \"why_RNNs\":why_RNNs,\n",
        "    \"why_not_RNNs\":why_not_RNNs,\n",
        "    \"creating_stationarity\":creating_stationarity,\n",
        "    \"why_pad\":why_pad,\n",
        "    \"cumulative_times\": times\n",
        "}\n",
        "\n",
        "src = \"https://airtable.com/embed/shrg2wXP3HvUi60OD?\"\n",
        "\n",
        "display(IFrame(src = prefill_form(src, fields), width = 800, height = 400))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oDE0MJbb5dLH"
      },
      "source": [
        "## Feedback\n",
        "How could this session have been better? How happy are you in your group? How do you feel right now?\n",
        "\n",
        "Feel free to use the embeded form below or use this link:\n",
        "<a target=\"_blank\" rel=\"noopener noreferrer\" href=\"https://airtable.com/shrNSJ5ECXhNhsYss\">https://airtable.com/shrNSJ5ECXhNhsYss</a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IPPjyA-H5kLE"
      },
      "source": [
        "display(IFrame(src=\"https://airtable.com/embed/shrNSJ5ECXhNhsYss?backgroundColor=red\", width = 800, height = 400))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}